
catalinaPreSize=0                   # 上次循环时,日志的大小
C_catalina(){
    local keyLines=0 catalinaLogSize=0 keyWord output tmplog=/tmp/catalina.log
    local tomcatPID=`ps -ef O+T |grep 'tomcat.*Bootstrap' |grep -v grep |head -n 1 |awk '{print $2}'`
    catalinaLogSize=`wc -c $1 |awk '{print $1}'`
    if (( catalinaPreSize < catalinaLogSize && c != 1 )); then                              # 当 catalinaPreSize > catalinaLogSize 时,日志可能被rotate了,此时也忽略其告警
        keyLines="$(tail -c"+${catalinaPreSize}" "$1" |grep -v ' DEBUG ' |egrep -c "$2")"   # keyLines 表示:含有报错关键字的行共有多少行
    fi                                                                                      # 当c==1时,catalinaLogSize可能很大,故忽略smon第1次循环时所产生的告警
    if [[ ${keyLines} != 0 ]]; then                                         
      keyWord="$(tail -c"+${catalinaPreSize}" "$1" |grep -v ' DEBUG ' |egrep -o "$2" |head -n 1)"
      output="$(echo -e "${awk_warn}: 发现 '${keyWord}' 等报错,请检查tomcat日志")"
    else
      output="$(echo "GOOD: tomcat日志没发现重大报错信息.")"
    fi
    Output
    if [[ $(echo "${output}" |grep -c WARN) != 0 ]]; then
        output="$(Eval_cont "ps -Lm up ${tomcatPID}"
        Eval_cont "jstack -l ${tomcatPID}" || Eval_cont "jstack -F -m -l ${tomcatPID}")"
        tail -c +${catalinaPreSize} ${1} > ${tmplog}
        MailToAny "${mail_to}" "${hostName}: [告警]tomcat日志有'${keyWord}'报错" "tomcat日志请查阅附件,下面列出tomcat的线程信息:
$(Output |Change_to_html)" "${tmplog}" && rm -f ${tmplog} &
    fi
    catalinaPreSize=${catalinaLogSize}
}

jvmGC_status=$GOOD
C_jvmGC(){
    local output output2 gc_file check_jvm fullGC tomcatPID tomcat_jstat heapdumpFileNumber
    gc_file=$(ls -1tr ${gc_files} |tail -n1)
    check_jvm="$(tail -n50 ${gc_file} |grep '\->'|tail -n1)"
    fullGC=$(echo ${check_jvm} | grep -c 'Full')
    tomcatPID=`ps -ef O+T |grep 'tomcat.*Bootstrap' |grep -v grep |head -n 1 |awk '{print $2}'` # 只获取一个进程号;若jstat $PID1 $PID2则将无限循环
    tomcat_jstat="$(jstat -gc ${tomcatPID})"                                                    # jstat -gc显示每种内存类型的使用量和使用率
    echo "${tomcat_jstat}" |tail -n 1 > ${zabbix_tmp}/jstat_tomcat_for_zabbix.tmp               # 把 GC 数据提供给 zabbix 使用
                            # 执行jstat需要时间,若直接执行jstat -gc ${tomcatPID} > ${zabbix_tmp}/jstat_tomcat_for_zabbix.tmp,则zabbix获取数据会失败
    output="$(if [[ ${fullGC} != 1 ]]; then
        echo "${check_jvm}" |awk -v v="$1" -v warn="${awk_warn}" -F'[][]|[K>]' '{x=$10/$8; 
        if(x<v)
            {print "GOOD: JVM usage is: " x*100 "% < thresold:" v*100 "%" }
        else 
            {print warn": JVM usage is: " x*100 "% > thresold:" v*100 "%" }
        }'
    else
        echo "${check_jvm}" |awk -v v="$1" -v warn="${awk_warn}" -F'[][]|[K>]' '{x=$16/$14;
        if(x<v)
            {print "GOOD: JVM usage is: " x*100 "%" < thresold:" v*100 "%" }
        else
            {print warn": JVM usage is: " x*100 "% > thresold:" v*100 "%" }
        }'
    fi)"
    Output
    if [[ $(echo "${output}" |grep -c WARN) != 0 && ${jvmGC_status} == $GOOD ]]; then
        output2="$(Eval_cont "ps -Lm up ${tomcatPID}  # 显示tomcat程序各个线程的CPU使用率"
        Eval_cont "jstack -l ${tomcatPID}" || Eval_cont "jstack -F -m -l ${tomcatPID}")"          # jstack命令显示栈信息
        MailToAny "${mail_to}" "${hostName}: [告警]JVM使用率超过 $1 * 100%" "$(Output |Change_to_html)
$(PInfo "正在抓取heapdump文件到/mnt目录,可以去检查一下.另外, heapdump 使用信息如下:"| Change_to_html)
${tomcat_jstat}<hr />$(echo "${output2}" |Change_to_html)" &
        jvmGC_status=$BAD
        # 抓取heapdump
        heapdumpFileNumber=`ls /mnt/heapdump*hprof |wc -l`
        ((heapdumpFileNumber>=2)) && rm $(ls -tr /mnt/heapdump*hprof|head -n 1)     # 删除最老的那个heapdump文件
        jmap -dump:file=/mnt/heapdump`date +%Y%m%d%H%M%S`.hprof ${tomcatPID} ||
            jmap -F -dump:file=/mnt/heapdump`date +%Y%m%d%H%M%S`.hprof ${tomcatPID} &
    elif [[ $(echo "${output}" |grep -c WARN) == 0 && ${jvmGC_status} == $BAD ]]; then
        MailToAny "${mail_to}" "${hostName}: [恢复]JVM使用率" "$(Output |Change_to_html)<br />${tomcat_jstat}<hr />$(Eval_cont "ls -lhtr /mnt/heapdump*"|Change_to_html)" &
        jvmGC_status=$GOOD
    fi
}

mysql_status=$GOOD
C_mysql_lock(){
    local lock_wait trx_count output i
    lock_wait=$(echo $1 |awk -F, '{print $1}')
    trx_count=$(echo $1 |awk -F, '{print $2}')
    output="$(mysql -u"${mysql_user}" -p"${mysql_password}" -h"${mysql_ip}" -N -e "(select count(time) from information_schema.processlist
where id in (select trx_mysql_thread_id from information_schema.INNODB_TRX where trx_id in
(select requesting_trx_id from information_schema.innodb_lock_waits)) and time>${lock_wait})
union all
(select count(*) as 'count(information_schema.innodb_trx)' from information_schema.innodb_trx
where trx_query is not null)" | awk -v c="${trx_count}" -v warn="${awk_warn}" -v t="${lock_wait}" '
(NR==1){ if ($1 > 0) print warn": 发现了"$1"个大于"t"秒的SQL锁"
else print "GOOD: 没发现大于"t"秒的SQL锁"}
(NR==2){ if ($1 > c) print warn": 事务数量超过了"c"个"
else print "GOOD: 事务数量不超过"c"个"}')"
    [[ ${output} == "" ]] && output="ERROR: SQL语句执行失败,可能是用户名或密码不对,请检查程序"
    Output
    if [[ $(echo "${output}" |grep -c WARN) != 0 && ${mysql_status} == $GOOD ]]; then
        output="${output}<hr />$(mysql -u"${mysql_user}" -p"${mysql_password}" -h"${mysql_ip}" -H -v -e '
                select * from information_schema.innodb_lock_waits order by blocking_trx_id; 
                select * from information_schema.innodb_trx order by trx_started desc; 
                select * from information_schema.processlist where info is not null order by info desc, time')"
        MailToAny "${mail_to}" "${hostName}: [告警]发现SQL锁或过多的事务数" "$(Output |Change_to_html)" &
        mysql_status=$BAD
    elif [[ $(echo "${output}" |grep -c WARN) == 0 && ${mysql_status} == $BAD ]]; then
        MailToAny "${mail_to}" "${hostName}: [恢复]SQL锁和事务数" "$(Output |Change_to_html)" &
        mysql_status=$GOOD
    fi
}


